{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 注意力模型实现中英文机器翻译\n",
    "\n",
    "### 1.数据预处理\n",
    "\n",
    "首先先下载本目录的数据和代码，并执行 **datautil.py**，生成中、英文字典\n",
    "\n",
    "### 2.执行如下代码\n",
    "\n",
    "训练时间会比较长\n",
    "\n",
    "### 3.测试模型\n",
    "运行 **test.py**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache /tmp/jieba.cache\n",
      "Loading model cost 0.657 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import os\n",
    "\n",
    "import sys\n",
    "import time\n",
    "import numpy as np\n",
    "from six.moves import xrange\n",
    "import tensorflow as tf\n",
    "datautil = __import__(\"datautil\")\n",
    "seq2seq_model = __import__(\"seq2seq_model\")\n",
    "import datautil\n",
    "import seq2seq_model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vocab_size 11963\n",
      "vocab_sizech 15165\n",
      "checkpoint_dir is fanyichina/checkpoints/\n",
      "WARNING:tensorflow:From /usr/local/python3/lib/python3.6/site-packages/tensorflow/python/framework/op_def_library.py:263: colocate_with (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Colocations handled automatically by placer.\n",
      "\n",
      "WARNING: The TensorFlow contrib module will not be included in TensorFlow 2.0.\n",
      "For more information, please see:\n",
      "  * https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md\n",
      "  * https://github.com/tensorflow/addons\n",
      "If you depend on functionality not listed there, please file an issue.\n",
      "\n",
      "WARNING:tensorflow:From /home/python_home/WeiZhongChuang/ML/TensorFlow/Attention/seq2seq_model.py:124: GRUCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.GRUCell, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:From /home/python_home/WeiZhongChuang/ML/TensorFlow/Attention/seq2seq_model.py:128: MultiRNNCell.__init__ (from tensorflow.python.ops.rnn_cell_impl) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "This class is equivalent as tf.keras.layers.StackedRNNCells, and will be replaced by that in Tensorflow 2.0.\n",
      "WARNING:tensorflow:At least two cells provided to MultiRNNCell are the same object and will share weights.\n",
      "new a cell\n",
      "WARNING:tensorflow:From /usr/local/python3/lib/python3.6/site-packages/tensorflow/contrib/legacy_seq2seq/python/ops/seq2seq.py:863: static_rnn (from tensorflow.python.ops.rnn) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `keras.layers.RNN(cell, unroll=True)`, which is equivalent to this API\n",
      "WARNING:tensorflow:From /usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/rnn_cell_impl.py:1259: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
      "WARNING:tensorflow:From /usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/nn_impl.py:1444: sparse_to_dense (from tensorflow.python.ops.sparse_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Create a `tf.sparse.SparseTensor` and use `tf.sparse.to_dense` instead.\n",
      "new a cell\n",
      "new a cell\n",
      "new a cell\n",
      "WARNING:tensorflow:From /usr/local/python3/lib/python3.6/site-packages/tensorflow/python/ops/array_grad.py:425: to_int32 (from tensorflow.python.ops.math_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use tf.cast instead.\n",
      "model is ok\n",
      "WARNING:tensorflow:From /usr/local/python3/lib/python3.6/site-packages/tensorflow/python/training/saver.py:1266: checkpoint_exists (from tensorflow.python.training.checkpoint_management) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "Use standard file APIs to check for files with this prefix.\n",
      "INFO:tensorflow:Restoring parameters from fanyichina/checkpoints/seq2seqtest.ckpt-54000\n",
      "Reading model parameters from fanyichina/checkpoints/seq2seqtest.ckpt-54000\n",
      "Using bucket sizes:\n",
      "[(20, 20), (40, 40), (50, 50), (60, 60)]\n",
      "fanyichina/fromids/english1w.txt\n",
      "fanyichina/toids/chinese1w.txt\n",
      "bucket sizes = [1649, 4933, 1904, 1383]\n",
      "global step 54200 learning rate 0.3699 step-time 0.72 perplexity 3.16\n",
      "fanyichina/checkpoints/seq2seqtest.ckpt\n",
      "  eval: bucket 0 perplexity 1.74\n",
      "输入 ['third', ',', 'the', 'pace', 'of', 'selling', 'public', 'houses', 'was', 'accelerated', '.', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD']\n",
      "输出 ['_GO', '三', '是', '加快', '了', '公有', '住房', '的', '出售', '.', '_EOS', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD']\n",
      "  eval: bucket 1 perplexity 2.93\n",
      "输入 ['thanks', 'to', 'the', 'equilibrium', 'of', 'the', 'international', 'payments', ',', 'china', \"'s\", 'exchange', 'rates', 'have', 'all', 'along', 'been', 'comparatively', 'stable', '.', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD']\n",
      "输出 ['_GO', '由', '於', '国际', '收支平衡', ',', '中国', '的', '汇率', '一直', '比较', '稳定', '.', '_EOS', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD']\n",
      "结果 ['记者', '中国', '关系', '进行', '对']\n",
      "  eval: bucket 2 perplexity 3.96\n",
      "输入 ['in', 'order', 'to', 'respond', 'to', 'the', 'vast', 'business', 'opportunities', 'in', 'the', 'future', 'when', 'there', 'are', 'direct', 'cross', '-', 'strait', 'flights', ',', 'taiwan', \"'s\", 'fu', 'hsing', 'aviation', 'has', 'spent', 'a', 'huge', 'sum', 'on', 'buying', 'medium', 'and', 'long', '-', 'range', 'versions', 'of', 'the', 'european', 'airbus', '.', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD']\n",
      "输出 ['_GO', '台湾', '复兴', '航空', '为', '因', '应', '未来', '两岸', '直航', '的', '庞大', '商机', ',', '大笔', '购', '进', '欧洲', '空中', '巴士', '的', '中', ',', '长程', '客机', '.', '_EOS', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD']\n",
      "结果 ['建设', '建设', '说', '建设', '加强', '是', '说', '是', '重要', '是', '是', '是', '实现', '市场', '这', '这', '的']\n",
      "  eval: bucket 3 perplexity 5.66\n",
      "输入 ['zhu', 'bangzao', 'said', 'that', 'after', 'hong', 'kong', \"'s\", 'reversion', ',', 'china', \"'s\", 'central', 'government', 'and', 'the', 'hong', 'kong', 'special', 'administrative', 'region', '[', 'hksar', ']', 'government', 'implemented', 'the', 'policy', 'of', '\"', 'one', 'country', ',', 'two', 'systems', ',', '\"', '\"', 'hong', 'kong', 'people', 'governing', 'hong', 'kong', ',', '\"', 'and', 'a', 'high', 'degree', 'of', 'autonomy', '.', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD']\n",
      "输出 ['_GO', '朱邦造', '说', ',', '香港', '回归', '后', ',', '中国', '中央政府', '和', '香港特区', '政府', '贯彻', '\"', '一国两制', '\"', '\"', '港人', '治', '港', '\"', '和', '高度', '自治', '的', '方针', '.', '_EOS', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD', '_PAD']\n"
     ]
    }
   ],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "steps_per_checkpoint=200 \n",
    "\n",
    "max_train_data_size= 0#(0: no limit)\n",
    "\n",
    "dropout = 0.9 \n",
    "grad_clip = 5.0\n",
    "batch_size = 60\n",
    "\n",
    "num_layers =2\n",
    "learning_rate =0.5\n",
    "lr_decay_factor =0.99\n",
    "\n",
    "###############翻译\n",
    "hidden_size = 100\n",
    "checkpoint_dir= \"fanyichina/checkpoints/\"\n",
    "\n",
    "_buckets =[(20, 20), (40, 40), (50, 50), (60, 60)]\n",
    "def getfanyiInfo():\n",
    "    vocaben, rev_vocaben=datautil.initialize_vocabulary(os.path.join(datautil.data_dir, datautil.vocabulary_fileen))\n",
    "    vocab_sizeen= len(vocaben)\n",
    "    print(\"vocab_size\",vocab_sizeen)\n",
    "    \n",
    "    vocabch, rev_vocabch=datautil.initialize_vocabulary(os.path.join(datautil.data_dir, datautil.vocabulary_filech))\n",
    "    vocab_sizech= len(vocabch)\n",
    "    print(\"vocab_sizech\",vocab_sizech) \n",
    "    \n",
    "    filesfrom,_=datautil.getRawFileList(datautil.data_dir+\"fromids/\")\n",
    "    filesto,_=datautil.getRawFileList(datautil.data_dir+\"toids/\")\n",
    "    source_train_file_path = filesfrom[0]\n",
    "    target_train_file_path= filesto[0]\n",
    "    return vocab_sizeen,vocab_sizech,rev_vocaben,rev_vocabch,source_train_file_path,target_train_file_path\n",
    "################################################################    \n",
    "#source_train_file_path = os.path.join(datautil.data_dir, \"data_source_test.txt\")\n",
    "#target_train_file_path = os.path.join(datautil.data_dir, \"data_target_test.txt\")    \n",
    "    \n",
    "\n",
    "def main():\n",
    "\t\n",
    "    vocab_sizeen,vocab_sizech,rev_vocaben,rev_vocabch,source_train_file_path,target_train_file_path = getfanyiInfo()\n",
    "\n",
    "    if not os.path.exists(checkpoint_dir):\n",
    "        os.mkdir(checkpoint_dir)\n",
    "    print (\"checkpoint_dir is {0}\".format(checkpoint_dir))\n",
    "\n",
    "    with tf.Session() as sess:\n",
    "        model = createModel(sess,False,vocab_sizeen,vocab_sizech)\n",
    "        print (\"Using bucket sizes:\")\n",
    "        print (_buckets)\n",
    "\n",
    "\n",
    "        source_test_file_path = source_train_file_path\n",
    "        target_test_file_path = target_train_file_path\n",
    "        \n",
    "        print (source_train_file_path)\n",
    "        print (target_train_file_path)\n",
    "        \n",
    "        train_set = readData(source_train_file_path, target_train_file_path,max_train_data_size)\n",
    "        test_set = readData(source_test_file_path, target_test_file_path,max_train_data_size)\n",
    "        \n",
    "        train_bucket_sizes = [len(train_set[b]) for b in xrange(len(_buckets))]\n",
    "        print( \"bucket sizes = {0}\".format(train_bucket_sizes))\n",
    "        train_total_size = float(sum(train_bucket_sizes))\n",
    "    \n",
    "        # A bucket scale is a list of increasing numbers from 0 to 1 that we'll use\n",
    "        # to select a bucket. Length of [scale[i], scale[i+1]] is proportional to\n",
    "        # the size if i-th training bucket, as used later.\n",
    "        train_buckets_scale = [sum(train_bucket_sizes[:i + 1]) / train_total_size for i in xrange(len(train_bucket_sizes))]\n",
    "        step_time, loss = 0.0, 0.0\n",
    "        current_step = 0\n",
    "        previous_losses = []\n",
    "        \n",
    "        while True:\n",
    "            # Choose a bucket according to data distribution. We pick a random number\n",
    "            # in [0, 1] and use the corresponding interval in train_buckets_scale.\n",
    "            random_number_01 = np.random.random_sample()\n",
    "            bucket_id = min([i for i in xrange(len(train_buckets_scale)) if train_buckets_scale[i] > random_number_01])\n",
    "\n",
    "            # 开始训练.\n",
    "            start_time = time.time()\n",
    "            encoder_inputs, decoder_inputs, target_weights = model.get_batch(train_set, bucket_id)\n",
    "            _, step_loss, _ = model.step(sess, encoder_inputs, decoder_inputs,target_weights, bucket_id, False)\n",
    "            step_time += (time.time() - start_time) / steps_per_checkpoint\n",
    "            loss += step_loss / steps_per_checkpoint\n",
    "            current_step += 1\n",
    "            \n",
    "            # 保存检查点，测试数据\n",
    "            if current_step % steps_per_checkpoint == 0:\n",
    "                # Print statistics for the previous epoch.\n",
    "                perplexity = math.exp(loss) if loss < 300 else float('inf')\n",
    "                print (\"global step %d learning rate %.4f step-time %.2f perplexity \"\n",
    "                    \"%.2f\" % (model.global_step.eval(), model.learning_rate.eval(),step_time, perplexity))\n",
    "                # Decrease learning rate if no improvement was seen over last 3 times.\n",
    "                if len(previous_losses) > 2 and loss > max(previous_losses[-3:]):\n",
    "                    sess.run(model.learning_rate_decay_op)\n",
    "                previous_losses.append(loss)\n",
    "                # Save checkpoint and zero timer and loss.\n",
    "                checkpoint_path = os.path.join(checkpoint_dir, \"seq2seqtest.ckpt\")\n",
    "                print(checkpoint_path)\n",
    "                model.saver.save(sess, checkpoint_path, global_step=model.global_step)\n",
    "                step_time, loss = 0.0, 0.0\n",
    "                # Run evals on development set and print their perplexity.\n",
    "                for bucket_id in xrange(len(_buckets)):\n",
    "                    if len(test_set[bucket_id]) == 0:\n",
    "                        print(\"  eval: empty bucket %d\" % (bucket_id))\n",
    "                        continue\n",
    "                    encoder_inputs, decoder_inputs, target_weights = model.get_batch(test_set, bucket_id)\n",
    "\n",
    "                    _, eval_loss,output_logits = model.step(sess, encoder_inputs, decoder_inputs,target_weights, bucket_id, True)\n",
    "                    eval_ppx = math.exp(eval_loss) if eval_loss < 300 else float('inf')\n",
    "                    print(\"  eval: bucket %d perplexity %.2f\" % (bucket_id, eval_ppx))\n",
    "                    \n",
    "                    \n",
    "                    inputstr = datautil.ids2texts(reversed([en[0] for en in encoder_inputs]) ,rev_vocaben)\n",
    "                    print(\"输入\",inputstr)\n",
    "                    print(\"输出\",datautil.ids2texts([en[0] for en in decoder_inputs] ,rev_vocabch))\n",
    "  \n",
    "                    outputs = [np.argmax(logit, axis=1)[0] for logit in output_logits]                    \n",
    "                    #outputs = [int(np.argmax(logit, axis=1)) for logit in output_logits]\n",
    "                    #print(\"outputs\",outputs,datautil.EOS_ID)\n",
    "                    if datautil.EOS_ID in outputs:\n",
    "                        outputs = outputs[:outputs.index(datautil.EOS_ID)]\n",
    "                        print(\"结果\",datautil.ids2texts(outputs,rev_vocabch))\n",
    "                        \n",
    "                    \n",
    "                    \n",
    "                sys.stdout.flush()\n",
    "\n",
    "\n",
    "def createModel(session, forward_only,from_vocab_size,to_vocab_size):\n",
    "    \"\"\"Create translation model and initialize or load parameters in session.\"\"\"\n",
    "    model = seq2seq_model.Seq2SeqModel(\n",
    "      from_vocab_size,#from\n",
    "      to_vocab_size,#to\n",
    "      _buckets,\n",
    "      hidden_size,\n",
    "      num_layers,\n",
    "      dropout,\n",
    "      grad_clip,\n",
    "      batch_size,\n",
    "      learning_rate,\n",
    "      lr_decay_factor,\n",
    "      forward_only=forward_only,\n",
    "      dtype=tf.float32)\n",
    "      \n",
    "    print(\"model is ok\")\n",
    "\n",
    "    \n",
    "    ckpt = tf.train.latest_checkpoint(checkpoint_dir)\n",
    "    if ckpt!=None:\n",
    "        model.saver.restore(session, ckpt)\n",
    "        print (\"Reading model parameters from {0}\".format(ckpt))\n",
    "    else:\n",
    "        print (\"Created model with fresh parameters.\")\n",
    "        session.run(tf.global_variables_initializer())  \n",
    "\n",
    "    return model\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "def readData(source_path, target_path, max_size=None):\n",
    "\t'''\n",
    "\tThis method directly from tensorflow translation example\n",
    "\t'''\n",
    "\tdata_set = [[] for _ in _buckets]\n",
    "\twith tf.gfile.GFile(source_path, mode=\"r\") as source_file:\n",
    "\t\twith tf.gfile.GFile(target_path, mode=\"r\") as target_file:\n",
    "\t\t\tsource, target = source_file.readline(), target_file.readline()\n",
    "\t\t\tcounter = 0\n",
    "\t\t\twhile source and target and (not max_size or counter < max_size):\n",
    "\t\t\t\tcounter += 1\n",
    "\t\t\t\tif counter % 100000 == 0:\n",
    "\t\t\t\t\tprint(\"  reading data line %d\" % counter)\n",
    "\t\t\t\t\tsys.stdout.flush()\n",
    "\t\t\t\tsource_ids = [int(x) for x in source.split()]\n",
    "\t\t\t\ttarget_ids = [int(x) for x in target.split()]\n",
    "\t\t\t\ttarget_ids.append(datautil.EOS_ID)\n",
    "\t\t\t\tfor bucket_id, (source_size, target_size) in enumerate(_buckets):\n",
    "\t\t\t\t\tif len(source_ids) < source_size and len(target_ids) < target_size:\n",
    "\t\t\t\t\t\tdata_set[bucket_id].append([source_ids, target_ids])\n",
    "\t\t\t\t\t\tbreak\n",
    "\t\t\t\tsource, target = source_file.readline(), target_file.readline()\n",
    "\treturn data_set\n",
    "\n",
    "\n",
    "\n",
    "if __name__ == '__main__':\n",
    "\tmain()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
